# -*- coding:utf-8 -*-
# --author： jingfeng 
# time: 2018/12/8


import multiprocessing  # 分布式进程
import multiprocessing.managers  # 分布式进程管理器
import random
import time
import queue

from lxml import etree
import requests
import queue
import re


def get_every_data(html):
    alllist = []
    mylist1 = []
    mylist2 = []

    mylist1 = get_all_http(html)
    try:
        if len(mylist1) > 1:
            mylist2 = getabsurl(mylist1, html)

            return mylist2

    except:
        pass


def getabsurl(url, html):
    try:
        regex = re.compile('href=\"(.*?)"')

        httplist = regex.findall(html)

        newttplist = httplist.copy()  # 深拷贝一份

        for data in newttplist:
            if data.find("http://") != -1:
                httplist.remove(data)

            if data.find("javascript") != -1:
                httplist.remove(data)

        host_name = gethostname(url)

        if host_name != None:
            for i in range(len(httplist)):
                httplist[i] = host_name + httplist[i]

        return httplist
    except Exception as e:
        print(e)


def gethostname(httpstr):
    try:
        emailregex = re.compile(r"(http://.*?)/")

        mylist = emailregex.findall(str(httpstr))

        if len(mylist) == 0:
            return None
        else:
            return mylist[0]

    except:
        return None


def get_all_http(html):
    try:
        find_url = re.findall(r"(http://\S*?)[\"|>|]", html)

        return find_url
    except:
        pass


def get_data(url):
    try:
        response = requests.get(url)
        html = response.text
        return html
    except Exception as e:
        pass


def get_all_email(html):
    try:

        email_reg = re.compile(r'([A-Z0-9a-z]+@[a-zA-z0-9.]+\.[A-Za-z0-9]{2,4})')
        email_list = re.findall(email_reg, html)
        return email_list
    except Exception as e:
        pass


class QueueManger(multiprocessing.managers.BaseManager):  # 继承，进程管理共享数据
    pass


if __name__ == '__main__':
    QueueManger.register('get_task')  # 注册两个函数，方便调用服务器
    QueueManger.register('get_result_email')
    QueueManger.register('get_result_url')

    manger = QueueManger(address=('192.168.1.174', 65034), authkey='000000'.encode('utf-8'))
    manger.connect()  # 连接服务器

    task = manger.get_task()
    result_email = manger.get_result_email()
    resulturl = manger.get_result_url()

    for i in range(1000):
        try:
            url = task.get()  # 从服务器得到url
            page_data = get_data(url)

            email_list = get_all_email(page_data)

            try:
                if len(email_list) != 0:
                    for email in email_list:
                        print(email)
                        result_email.put(email)

            except:
                pass

            urllist = get_every_data(page_data)
            try:
                if len(urllist) != 0:
                    for myurl in urllist:

                        resulturl.put(myurl)
            except:
                pass


        except:
            pass
